{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-aa5b6903d61d>:15: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n"
     ]
    }
   ],
   "source": [
    "#把所有数据整合到一起，list1是数据，list2是二分类标签，list10是10分类标签。\n",
    "#Put all the data together, list1 is the data, list2 is the dichotomy label, and list10 is the 10-category label.\n",
    "from collections import Counter\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "#读取tf文件数据\n",
    "#Read the tf file data\n",
    "list1= []#data\n",
    "list2=[]   #label_2\n",
    "list10=[]#label_10\n",
    "\n",
    "for serialized_example in tf.python_io.tf_record_iterator(\"E:/IDS/normalized/train_select_12.tfrecords\"):  \n",
    "\n",
    "\n",
    "    example = tf.train.Example()\n",
    "    example.ParseFromString(serialized_example)\n",
    "    \n",
    "\n",
    "    feature = example.features.feature['features'].float_list.value  \n",
    "    label_2 = example.features.feature['label_2'].float_list.value\n",
    "    label_10 = example.features.feature['label_10'].float_list.value\n",
    "    \n",
    "    list1.append(feature)  \n",
    "    list2.append(label_2)\n",
    "    list10.append(label_10)\n",
    "\n",
    "    \n",
    "# for serialized_example in tf.python_io.tf_record_iterator(\"/home/hll/IDS/normalized/test_select_12.tfrecords\"):  \n",
    "\n",
    "\n",
    "#     example = tf.train.Example()\n",
    "#     example.ParseFromString(serialized_example)\n",
    "    \n",
    "\n",
    "#     feature = example.features.feature['features'].float_list.value  \n",
    "#     label_2 = example.features.feature['label_2'].float_list.value\n",
    "#     label_10 = example.features.feature['label_10'].float_list.value\n",
    "    \n",
    "#     list1.append(feature)  \n",
    "#     list2.append(label_2)\n",
    "#     list10.append(label_10)\n",
    "    \n",
    "\n",
    "        \n",
    "# for serialized_example in tf.python_io.tf_record_iterator(\"/home/hll/IDS/normalized/validation_select_12.tfrecords\"):  \n",
    "\n",
    "\n",
    "#     example = tf.train.Example()\n",
    "#     example.ParseFromString(serialized_example)\n",
    "    \n",
    "\n",
    "#     feature = example.features.feature['features'].float_list.value  \n",
    "#     label_2 = example.features.feature['label_2'].float_list.value\n",
    "#     label_10 = example.features.feature['label_10'].float_list.value\n",
    "    \n",
    "#     list1.append(feature) \n",
    "#     list2.append(label_2)\n",
    "#     list10.append(label_10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=np.array(list1) \n",
    "b2=np.array(list2)\n",
    "bb2=b2.reshape(b2.shape[0],)     \n",
    "b10=np.array(list10)\n",
    "bb10=b10.reshape(b10.shape[0],)\n",
    "y2 = np.int32(bb2)  \n",
    "y10 = np.int32(bb10)\n",
    "y10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({0: 1553132, 1: 224898})\n",
      "Counter({0: 1553132, 6: 150836, 4: 31167, 5: 16972, 3: 11449, 7: 9791, 1: 1874, 2: 1630, 8: 1057, 9: 122})\n"
     ]
    }
   ],
   "source": [
    "#View the distribution of data classes (data distribution without imbalanced processing)\n",
    "#查看数据类的分布(未不平衡处理的数据分布)\n",
    "print(Counter(y2))  \n",
    "print(Counter(y10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 238.82638692855835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0, 1548999), (1, 1548999)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "import time\n",
    "time_start = time.time()\n",
    "smo = SMOTE(random_state=42) \n",
    "\n",
    "X_smo, y_smo = smo.fit_sample(X_tl2, y_tl2)   \n",
    "\n",
    "time_end = time.time()\n",
    "time = time_end - time_start\n",
    "print(\"time:\",time)\n",
    "\n",
    "sorted(Counter(y_smo).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3097998, 1)\n"
     ]
    }
   ],
   "source": [
    "yy2 = y_smo.reshape(y_smo.shape[0],1)  \n",
    "print(yy2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the data as an npy file\n",
    "#把数据存成npy格式的文件\n",
    "np.save(\"E:/IDS/alldata/12/tomektrain/data.npy\",X_smo)  \n",
    "np.save(\"E:/IDS/alldata/12/tomektrain/label_2.npy\",yy2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# yy2 = y2.reshape(y2.shape[0],1)  #将标签再重塑为列向量形式,最终分类需要这种格式\n",
    "# print(yy2.shape)\n",
    "# yy10 = y10.reshape(y10.shape[0],1)  #将标签再重塑为列向量形式,最终分类需要这种格式\n",
    "# print(yy10.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.save(\"/home/hll/IDS/alldata/12/val/data.npy\",X)  #把数据存成npy格式的文件\n",
    "# np.save(\"/home/hll/IDS/alldata/12/val/label_2.npy\",yy2)\n",
    "# np.save(\"/home/hll/IDS/alldata/12/val/label_10.npy\",yy10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test set does not do processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存测试集数据\n",
    "#Save the test set data\n",
    "list11= []#data\n",
    "list22=[]   #label_2\n",
    "list1010=[]#label_10\n",
    "\n",
    "for serialized_example in tf.python_io.tf_record_iterator(\"normalized/test_select_12.tfrecords\"):  \n",
    "\n",
    "    example = tf.train.Example()\n",
    "    example.ParseFromString(serialized_example)\n",
    "\n",
    "    feature = example.features.feature['features'].float_list.value  \n",
    "    label_2 = example.features.feature['label_2'].float_list.value\n",
    "    label_10 = example.features.feature['label_10'].float_list.value\n",
    "    \n",
    "    list11.append(feature) \n",
    "    list22.append(label_2)\n",
    "    list1010.append(label_10)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XX=np.array(list11) \n",
    "b22=np.array(list22)\n",
    "bb22=b22.reshape(b22.shape[0],)   \n",
    "b1010=np.array(list1010)\n",
    "bb1010=b1010.reshape(b1010.shape[0],)\n",
    "y22 = np.int32(bb22)  \n",
    "y1010 = np.int32(bb1010)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y22.shape)\n",
    "print(y1010.shape)  \n",
    "print(XX.shape)\n",
    "print(Counter(y22))  \n",
    "print(Counter(y1010))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy22 = y22.reshape(y22.shape[0],1)  \n",
    "print(yy22.shape)\n",
    "yy1010 = y1010.reshape(y1010.shape[0],1)  \n",
    "print(yy1010.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"alldata/data_12_test.npy\",XX) \n",
    "np.save(\"alldata/label_2_12_test.npy\",yy22)\n",
    "np.save(\"alldata/label_10_12_test.npy\",yy1010)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROS-binary-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "ros = RandomOverSampler(random_state=0)\n",
    "X_ros, y_ros = ros.fit_sample(X, y2)\n",
    "\n",
    "sorted(Counter(y_ros).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy2 = y_ros.reshape(y_ros.shape[0],1)  \n",
    "yy2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"/home/hll/IDS/cicdata/77_2/data_ros_train.npy\",X_ros) \n",
    "np.save(\"/home/hll/IDS/cicdata/77_2/label_ros_train.npy\",yy2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROS-multi-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "ros = RandomOverSampler(random_state=0)\n",
    "X_ros, y_ros = ros.fit_sample(X, y10)\n",
    "\n",
    "sorted(Counter(y_ros).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy10 = y_ros.reshape(y_ros.shape[0],1)  \n",
    "yy10.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"/home/hll/IDS/alldata/12/rostrain/data_ros10.npy\",X_ros) \n",
    "np.save(\"/home/hll/IDS/alldata/12/rostrain/label_10_ros.npy\",yy10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMOTE-binary-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "smo = SMOTE(random_state=42)  \n",
    "\n",
    "X_smo, y_smo = smo.fit_sample(X, y2)    \n",
    "sorted(Counter(y_smo).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy2 = y_smo.reshape(y_smo.shape[0],1) \n",
    "yy2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"/home/hll/IDS/cicdata/77_2/data_smote_train.npy\",X_smo) \n",
    "np.save(\"/home/hll/IDS/cicdata/77_2/label_smote_train.npy\",yy2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMOTE-multi-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "smo = SMOTE(random_state=42)  \n",
    "\n",
    "X_smo, y_smo = smo.fit_sample(X, y10)   \n",
    "sorted(Counter(y_smo).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy10 = y_smo.reshape(y_smo.shape[0],1) \n",
    "yy10.shape\n",
    "np.save(\"/home/hll/IDS/alldata/12/smotetrain/data_12smo10.npy\",X_smo)  \n",
    "np.save(\"/home/hll/IDS/alldata/12/smotetrain/label_10_12smo.npy\",yy10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy10.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADASYN-binary-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import  ADASYN\n",
    "X_adasyn, y_adasyn = ADASYN().fit_sample(X, y2)\n",
    "\n",
    "print(sorted(Counter(y_adasyn).items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy2 = y_adasyn.reshape(y_adasyn.shape[0],1)  \n",
    "yy2.shape\n",
    "np.save(\"/home/hll/IDS/cicdata/77_2/data_adasyn_train.npy\",X_adasyn) \n",
    "np.save(\"/home/hll/IDS/cicdata/77_2/label_adasyn_train.npy\",yy2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADASYN-multi-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import  ADASYN\n",
    "X_adasyn, y_adasyn = ADASYN(ratio={1:10000,2:10000,3:100000,4:100000,\n",
    "                                   5:100000,6:1000000,7:10000,8:10000,9:1000},random_state=42).fit_sample(X, y10)\n",
    "\n",
    "print(sorted(Counter(y_adasyn).items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy2 = y_adasyn.reshape(y_adasyn.shape[0],1)  \n",
    "yy2.shape\n",
    "np.save(\"alldata/ADASYN/data_12adasyn10.npy\",X_adasyn)  \n",
    "np.save(\"alldata/ADASYN/label_10_12adasyn.npy\",yy2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
